package study.test.crawler;

import java.util.List;

import org.apache.commons.collections.CollectionUtils;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

/**
 * 练习爬取文章 爬取自己的博客的所有文章的标题
 * 
 * @ClassName CSDNPageProcessor
 * @author <a href="892042158@qq.com" target="_blank">于国帅</a>
 * @date 2019年3月23日 上午11:01:30
 *
 */
public class CSDNPageProcessor implements PageProcessor {
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);
    int count = 1;

    @Override
    public void process(Page page) {

        Html html = page.getHtml();
        // 开始获取列表
        Selectable articleListSelectable = html.$(".article-list");
        // 筛选对应的列表
        List<Selectable> list = articleListSelectable.$(".article-item-box.csdn-tracking-statistics").nodes();
        // 输出所有列表的标题
        if (CollectionUtils.isNotEmpty(list)) {
            for (Selectable selectable : list) {
                String title = selectable.$("h4 a").xpath("a/text()").toString();
                System.err.println("抓取到的标题" + title);
            }
            System.err.println("总共抓取到了" + list.size() + "篇文章");
            // 如果数量大于20 继续爬取
            if (list.size() >= 20) {
                String url = page.getUrl().toString();
                System.err.println(url);
                url = "https://blog.csdn.net/mengxiangxingdong/article/list/" + count++;
                page.addTargetRequest(url);

            } else {
                System.err.println("抓取完毕");
            }
        }

    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        // 添加种子进行执行
        Spider.create(new CSDNPageProcessor()).addUrl("https://blog.csdn.net/mengxiangxingdong/article/list/").thread(5).run();

    }

}
